<html><!-- Created using the cpp_pretty_printer from the dlib C++ library.  See http://dlib.net for updates. --><head><title>dlib C++ Library - dnn.cpp</title></head><body bgcolor='white'><pre>
<font color='#009900'>// Copyright (C) 2015  Davis E. King (davis@dlib.net)
</font><font color='#009900'>// License: Boost Software License   See LICENSE.txt for the full license.
</font>

<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>sstream<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>string<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>cstdlib<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>ctime<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>vector<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../dnn.h.html'>../dnn.h</a>"

<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='tester.h.html'>tester.h</a>"

<font color='#0000FF'>namespace</font>
<b>{</b>

    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> test;
    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib;
    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;

    logger <b><a name='dlog'></a>dlog</b><font face='Lucida Console'>(</font>"<font color='#CC0000'>test.dnn</font>"<font face='Lucida Console'>)</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>float</u></font> <b><a name='compare_gradients'></a>compare_gradients</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> t,
        T grad
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'><u>float</u></font> max_error <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#0000FF'>auto</font> p <font color='#5555FF'>=</font> t.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> t.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            max_error <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>max_error, std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>p[i]<font color='#5555FF'>-</font><font color='#BB00BB'>grad</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'>return</font> max_error;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_tanh'></a>test_tanh</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor src, dest, gradient_input;
        src <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dest <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        gradient_input <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;



        <font color='#0000FF'>auto</font> grad_src <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>tanh</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;

        resizable_tensor src_grad;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        <font color='#BB00BB'>tanh</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>tanh_gradient</font><font face='Lucida Console'>(</font>src_grad, dest, gradient_input<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font> grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>src_grad, grad_src<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_sigmoid'></a>test_sigmoid</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor src, dest, gradient_input;
        src <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dest <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        gradient_input <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;



        <font color='#0000FF'>auto</font> grad_src <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>sigmoid</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;

        resizable_tensor src_grad;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        <font color='#BB00BB'>sigmoid</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>sigmoid_gradient</font><font face='Lucida Console'>(</font>src_grad, dest, gradient_input<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font> grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>src_grad, grad_src<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_softmax'></a>test_softmax</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,nr,nr<font face='Lucida Console'>)</font>, <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>gradient_input</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,nr,nc<font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        <font color='#009900'>// fill like this as a test of the assignment operator.
</font>        gradient_input <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font><font color='#5555FF'>*</font>nr<font color='#5555FF'>*</font>nc, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;



        <font color='#0000FF'>auto</font> grad_src <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                tt::<font color='#BB00BB'>softmax</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;

        resizable_tensor src_grad;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        tt::<font color='#BB00BB'>softmax</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>softmax_gradient</font><font face='Lucida Console'>(</font>src_grad, dest, gradient_input<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font> grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>src_grad, grad_src<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_batch_normalize'></a>test_batch_normalize</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor src, gamma, beta, dest, dest2, dest3, means, vars, gradient_input;
        src <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        gamma <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>5</font>, <font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        beta <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>5</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        gradient_input <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>3</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        gamma <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        beta <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        resizable_tensor running_means;
        resizable_tensor running_variances;
        <font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1.0</font><font face='Lucida Console'>)</font>;
        <font color='#009900'>// Turn back into biased variance estimate because that's how batch_normalize() works, so if we want to match it this is necessary.
</font>        running_variances <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale; 
        <font color='#BB00BB'>batch_normalize_inference</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest2, src, gamma, beta, running_means, running_variances<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>batch_normalize_inference</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest3, src, gamma, beta, running_means, running_variances<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest3<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest3<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;


        <font color='#0000FF'>auto</font> grad_src <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;
        <font color='#0000FF'>auto</font> grad_gamma <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;
        <font color='#0000FF'>auto</font> grad_beta <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;

        resizable_tensor src_grad, gamma_grad, beta_grad;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        gamma_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>gamma<font face='Lucida Console'>)</font>;
        beta_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>beta<font face='Lucida Console'>)</font>;
        src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>8</font>;
        beta_grad <font color='#5555FF'>=</font> <font color='#979000'>8</font>;

        <font color='#BB00BB'>batch_normalize_gradient</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font> grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>src_grad, grad_src<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;

        grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>gamma_grad, grad_gamma<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>gamma error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;

        grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>beta_grad, grad_beta<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>beta error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_batch_normalize_conv'></a>test_batch_normalize_conv</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>, gamma, beta, dest, dest2, dest3, means, vars, <font color='#BB00BB'>gradient_input</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>gradient_input<font face='Lucida Console'>)</font>;
        gamma <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>5</font>, <font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        beta <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>5</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        gamma <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        beta <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        resizable_tensor running_means;
        resizable_tensor running_variances;
        <font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1.0</font><font face='Lucida Console'>)</font>;
        <font color='#009900'>// Turn back into biased variance estimate because that's how
</font>        <font color='#009900'>// batch_normalize_conv() works, so if we want to match it this is necessary.
</font>        running_variances <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale; 
        <font color='#BB00BB'>batch_normalize_conv_inference</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest2, src, gamma, beta, running_means, running_variances<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>batch_normalize_conv_inference</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest3, src, gamma, beta, running_means, running_variances<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest3<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;


        <font color='#0000FF'>auto</font> grad_src <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;
        <font color='#0000FF'>auto</font> grad_gamma <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;
        <font color='#0000FF'>auto</font> grad_beta <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;


        resizable_tensor src_grad, gamma_grad, beta_grad;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        gamma_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>gamma<font face='Lucida Console'>)</font>;
        beta_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>beta<font face='Lucida Console'>)</font>;
        src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>9</font>;
        beta_grad <font color='#5555FF'>=</font> <font color='#979000'>9</font>;

        <font color='#BB00BB'>batch_normalize_conv_gradient</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad<font face='Lucida Console'>)</font>;


        <font color='#0000FF'>auto</font> grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>src_grad, grad_src<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;

        grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>gamma_grad, grad_gamma<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>gamma error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;

        grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>beta_grad, grad_beta<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>beta error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;

    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_basic_tensor_ops'></a>test_basic_tensor_ops</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor dest, <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>A</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>B</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        src <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        dest.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, <font color='#979000'>2</font>, <font color='#979000'>3</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>truth1</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>truth2</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;

        truth1 <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        truth2 <font color='#5555FF'>=</font> <font color='#979000'>7</font>, <font color='#979000'>10</font>,  <font color='#979000'>7</font>,  <font color='#979000'>7</font>,
        <font color='#979000'>7</font>, <font color='#979000'>10</font>,  <font color='#979000'>7</font>,  <font color='#979000'>7</font>,
        <font color='#979000'>7</font>, <font color='#979000'>10</font>,  <font color='#979000'>7</font>,  <font color='#979000'>7</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth1<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        A <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        B <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>1</font>] <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>1</font>] <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        dest <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, A, B<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth2<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        A <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font>, <font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        B <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, A, B<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> truth3 <font color='#5555FF'>=</font> <font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth3<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> truth4 <font color='#5555FF'>=</font> <font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, A, A, B<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth4<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        truth4 <font color='#5555FF'>=</font> <font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
        tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, A, A, B<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth4<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> truth5 <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.1</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> truth5;
        <font color='#BB00BB'>threshold</font><font face='Lucida Console'>(</font>B, <font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth5<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'><u>int</u></font> cnt <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#0000FF'>for</font><font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> x : A<font face='Lucida Console'>)</font>
            x <font color='#5555FF'>=</font> cnt<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;

        truth1.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        truth2.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        truth3.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        truth1 <font color='#5555FF'>=</font> <font color='#979000'>0</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font>,<font color='#979000'>3</font>;
        truth2 <font color='#5555FF'>=</font> <font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>6</font>,<font color='#979000'>7</font>;
        truth3 <font color='#5555FF'>=</font> <font color='#979000'>8</font>,<font color='#979000'>9</font>,<font color='#979000'>10</font>,<font color='#979000'>11</font>;

        alias_tensor <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font> A0 <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font> A4 <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font> A8 <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font><font color='#0000FF'>const_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>const</font> resizable_tensor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,<font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A0<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> truth1<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font><font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>4</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> truth2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A8<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> truth3<font face='Lucida Console'>)</font>;

        A4 <font color='#5555FF'>+</font><font color='#5555FF'>=</font> uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        truth2 <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A4<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> truth2<font face='Lucida Console'>)</font>;
        truth1 <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font><font color='#BB00BB'>reshape_to_column_vector</font><font face='Lucida Console'>(</font>truth1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        truth2 <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font><font color='#BB00BB'>reshape_to_column_vector</font><font face='Lucida Console'>(</font>truth2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        truth3 <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font><font color='#BB00BB'>reshape_to_column_vector</font><font face='Lucida Console'>(</font>truth3<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font>truth1,<font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font>truth2,truth3<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>A,A,<font color='#979000'>1</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        truth1 <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        truth2 <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        truth3 <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font><font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>4</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>reshape</font><font face='Lucida Console'>(</font>truth2,<font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font>truth1,<font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font>truth2,truth3<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <b>{</b>
            resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            resizable_tensor A, B;
            A <font color='#5555FF'>=</font> dest;
            B <font color='#5555FF'>=</font> dest;

            tensor_rand rnd;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;

            dest.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;

            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> AA <font color='#5555FF'>=</font> <font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; AA <font color='#5555FF'>=</font> <font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,AA<font face='Lucida Console'>)</font>;

            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> prevdest <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>prevdest<font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            dest.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
            prevdest <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>prevdest<font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
            prevdest <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>prevdest<font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
        <b>}</b>

        <b>{</b>
            resizable_tensor A, B, truth;
            A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            truth.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            B <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            truth <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>A, truth<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>A, truth<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>A, truth<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>A, truth<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>A, truth<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>host_write_only</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>A, truth<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
        <b>}</b>

        <b>{</b>
            resizable_tensor A, B;
            A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>11</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            B <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> truth;


            alias_tensor <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>{</b>
                <font color='#009900'>// non-aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>B,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,  <font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <b>{</b>
                <font color='#009900'>// aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,  <font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>


<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>{</b>
                <font color='#009900'>// non-aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>B,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,  <font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <b>{</b>
                <font color='#009900'>// aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,  <font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>


            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>{</b>
                <font color='#009900'>// non-aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>B,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,  <font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <b>{</b>
                <font color='#009900'>// aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,  <font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>{</b>
                <font color='#009900'>// non-aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>B,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,  <font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <b>{</b>
                <font color='#009900'>// aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,  <font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>

<font color='#0000FF'>#endif</font>
        <b>}</b>

        <b>{</b>
            resizable_tensor A, B;
            A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            tensor_rand rnd;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;

            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> truth;

            truth <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, A, <font color='#979000'>3</font>, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;


            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            truth <font color='#5555FF'>=</font> <font color='#979000'>0</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, A, <font color='#979000'>3</font>, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;

            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            truth <font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>0</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, A, <font color='#979000'>0</font>, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;


            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            truth <font color='#5555FF'>=</font> <font color='#979000'>0</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>0</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, A, <font color='#979000'>0</font>, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;


            B.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            truth <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, A, <font color='#979000'>3</font>, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>2</font><font face='Lucida Console'>)</font>;

            B.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> temp <font color='#5555FF'>=</font> <font color='#BB00BB'>join_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>, <font color='#BB00BB'>join_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            truth <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font>temp,temp<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, A, <font color='#979000'>3</font>, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            B.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>AA</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>, <font color='#BB00BB'>BB</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, A, <font color='#979000'>3</font>, B<font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, AA, <font color='#979000'>3</font>, BB<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>AA<font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>AA<font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA

    <font color='#0000FF'><u>void</u></font> <b><a name='test_conv'></a>test_conv</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        cuda::tensor_conv conv1;
        cpu::tensor_conv conv2;

        dlib::rand prnd;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>400</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            resizable_tensor <font color='#BB00BB'>data</font><font face='Lucida Console'>(</font>prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>5</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>5</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>25</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>25</font><font color='#5555FF'>+</font><font color='#979000'>1</font>
            <font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>
                prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>5</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>6</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>6</font><font color='#5555FF'>+</font><font color='#979000'>1</font> 
            <font face='Lucida Console'>)</font>;

            tt::tensor_rand rnd;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>filters<font face='Lucida Console'>)</font>;


            resizable_tensor output1, output2;


            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> stride_y <font color='#5555FF'>=</font> prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>5</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> stride_x <font color='#5555FF'>=</font> prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>5</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;
            <font color='#0000FF'><u>int</u></font> padding_y <font color='#5555FF'>=</font> prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>int</u></font> padding_x <font color='#5555FF'>=</font> prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                padding_y <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                padding_x <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            <font color='#BB00BB'>conv1</font><font face='Lucida Console'>(</font>output1, data, filters, stride_y,stride_x, padding_y, padding_x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>conv2</font><font face='Lucida Console'>(</font>output2, data, filters, stride_y,stride_x, padding_y, padding_x<font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>forward error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>3</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                 <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t padding_y: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> padding_y 
                 <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t padding_x: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> padding_x 
                 <font face='Lucida Console'>)</font>;



            resizable_tensor gi, data_gradient1, data_gradient2;
            gi.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>gi<font face='Lucida Console'>)</font>;

            data_gradient1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;
            data_gradient2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;
            data_gradient1 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            data_gradient2 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;

            conv1.<font color='#BB00BB'>get_gradient_for_data</font><font face='Lucida Console'>(</font>gi, filters, data_gradient1<font face='Lucida Console'>)</font>;
            conv2.<font color='#BB00BB'>get_gradient_for_data</font><font face='Lucida Console'>(</font>gi, filters, data_gradient2<font face='Lucida Console'>)</font>;

            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>data gradient error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>3</font><font face='Lucida Console'>)</font>;


            resizable_tensor filter_gradient1, filter_gradient2;
            gi.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>gi<font face='Lucida Console'>)</font>;

            filter_gradient1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>filters<font face='Lucida Console'>)</font>;
            filter_gradient2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>filters<font face='Lucida Console'>)</font>;
            filter_gradient1 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            filter_gradient2 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;

            conv1.<font color='#BB00BB'>get_gradient_for_filters</font><font face='Lucida Console'>(</font>gi, data, filter_gradient1<font face='Lucida Console'>)</font>;
            conv2.<font color='#BB00BB'>get_gradient_for_filters</font><font face='Lucida Console'>(</font>gi, data, filter_gradient2<font face='Lucida Console'>)</font>;

            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>filter gradient error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>3</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='compare_adam'></a>compare_adam</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'><u>float</u></font> t <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        tt::tensor_rand rnd;
        resizable_tensor s, m, v, params, params_grad;
        s.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>89</font>,<font color='#979000'>90</font>,<font color='#979000'>60</font>,<font color='#979000'>73</font><font face='Lucida Console'>)</font>;
        m.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;
        v.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;
        params.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;
        params_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;

        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>v<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>params<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>params_grad<font face='Lucida Console'>)</font>;

        resizable_tensor <font color='#BB00BB'>mm</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font>, <font color='#BB00BB'>vv</font><font face='Lucida Console'>(</font>v<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>compute_adam_update</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,s, mm, vv, t, <font color='#979000'>0.01</font>, <font color='#979000'>0.001</font>, <font color='#979000'>0.9</font>, <font color='#979000'>0.99</font>, params, params_grad<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> s1 <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;
        
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>compute_adam_update</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,s, m, v, t, <font color='#979000'>0.01</font>, <font color='#979000'>0.001</font>, <font color='#979000'>0.9</font>, <font color='#979000'>0.99</font>, params, params_grad<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> s2 <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>s1<font color='#5555FF'>-</font>s2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>s1<font color='#5555FF'>-</font>s2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>mm<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>mm<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>v<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>vv<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>v<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>vv<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_add'></a>test_add</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        dlib::rand rnd;
        tt::tensor_rand trnd;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>300</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            resizable_tensor <font color='#BB00BB'>dest1</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            resizable_tensor dest2;
            dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>src1</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>src2</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>dest1, src1, src2<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>dest2, src1, src2<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#009900'>// make sure we have a test for the case where all tensors have the same
</font>        <font color='#009900'>// dimensions.
</font>        resizable_tensor <font color='#BB00BB'>dest1</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>6</font><font face='Lucida Console'>)</font>;
        resizable_tensor dest2;
        resizable_tensor src1;
        resizable_tensor src2;
        dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
        src1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
        src2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;

        trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
        trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
        trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
        trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;

        cpu::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>dest1, src1, src2<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>dest2, src1, src2<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_more_ops'></a>test_more_ops</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr, <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#009900'>// We are going to make sure that the CPU implementation of these things matches
</font>        <font color='#009900'>// the CUDA implementation.
</font>
        tensor_rand rnd;

        resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, dest2, src2;
        resizable_tensor <font color='#BB00BB'>srcb</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>srcc</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, srcb2, srcc2;


        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest; src2 <font color='#5555FF'>=</font> src;
        cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, dest, src<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest2, dest2, src2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, dest, src<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest2, dest2, src2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;


        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest; src2 <font color='#5555FF'>=</font> src;
        cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, <font color='#979000'>2</font>, <font color='#979000'>3</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest2, src2, <font color='#979000'>2</font>, <font color='#979000'>3</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcb<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest; src2 <font color='#5555FF'>=</font> src; srcb2 <font color='#5555FF'>=</font> srcb;
        cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, srcb, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest2, src2, srcb2, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcb<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcc<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest; src2 <font color='#5555FF'>=</font> src; srcb2 <font color='#5555FF'>=</font> srcb; srcc2 <font color='#5555FF'>=</font> srcc;
        cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, srcb, srcc, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font>, <font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest2, src2, srcb2, srcc2, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font>, <font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, srcb, srcc, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest2, src2, srcb2, srcc2, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        cuda::<font color='#BB00BB'>affine_transform_range</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest, src, srcb, srcc, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform_range</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, dest2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest2, src2, srcb2, srcc2, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#979000'>3</font> <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            dest <font color='#5555FF'>=</font> <font color='#979000'>999</font>;
            dest2 <font color='#5555FF'>=</font> <font color='#979000'>999</font>;
            cuda::<font color='#BB00BB'>affine_transform_range</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>, dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font>, dest, src, srcb, srcc, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>affine_transform_range</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>, dest2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font>, dest2, src2, srcb2, srcc2, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            cuda::<font color='#BB00BB'>affine_transform_range</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest, src, srcb, srcc, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>affine_transform_range</font><font face='Lucida Console'>(</font>dest2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest2, src2, srcb2, srcc2, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>


        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcb<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcc<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest; src2 <font color='#5555FF'>=</font> src; srcb2 <font color='#5555FF'>=</font> srcb; srcc2 <font color='#5555FF'>=</font> srcc;
        cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, srcb, srcc<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest2, src2, srcb2, srcc2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#009900'>// now exercise code path where the A/B tensors have num_samples()==1
</font>        srcb.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,nc<font face='Lucida Console'>)</font>;
        srcc.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,nc<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcb<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcc<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest; src2 <font color='#5555FF'>=</font> src; srcb2 <font color='#5555FF'>=</font> srcb; srcc2 <font color='#5555FF'>=</font> srcc;
        cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, srcb, srcc<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest2, src2, srcb2, srcc2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;


        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        src2 <font color='#5555FF'>=</font> src;
        cuda::<font color='#BB00BB'>threshold</font><font face='Lucida Console'>(</font>src, <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>threshold</font><font face='Lucida Console'>(</font>src2, <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <b>{</b>
            resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            resizable_tensor A, B;
            A <font color='#5555FF'>=</font> dest;
            B <font color='#5555FF'>=</font> dest;

            tensor_rand rnd;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;

            dest.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;

            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; 

            A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> AA <font color='#5555FF'>=</font> <font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; AA <font color='#5555FF'>=</font> <font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,AA<font face='Lucida Console'>)</font>;

            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> prevdest <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>prevdest<font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            dest.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
            prevdest <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>prevdest<font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='compare_bn_gpu_and_cpu'></a>compare_bn_gpu_and_cpu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor dest, dest2;
        resizable_tensor means, means2;
        resizable_tensor invstds, invstds2;
        resizable_tensor running_means, running_means2;
        resizable_tensor running_variances, running_variances2;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font><font color='#979000'>64</font>,<font color='#979000'>20</font>,<font color='#979000'>100</font>,<font color='#979000'>100</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>20</font>,<font color='#979000'>100</font>,<font color='#979000'>100</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>20</font>,<font color='#979000'>100</font>,<font color='#979000'>100</font><font face='Lucida Console'>)</font>;
        gamma <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        beta <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;


        cpu::<font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, invstds, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest2,means2,invstds2, <font color='#979000'>1</font>, running_means2, running_variances2, src, gamma, beta<font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>dest error:    </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>means error:   </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>invstds error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>running_means error:   </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>running_variances error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font>,
            <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> 
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <font face='Lucida Console'>)</font>;


        <font color='#009900'>// now check that the gradients match as well
</font>        resizable_tensor gradient_input;
        resizable_tensor src_grad, gamma_grad, beta_grad;
        resizable_tensor src_grad2, gamma_grad2, beta_grad2;
        gradient_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>; src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>; src_grad2 <font color='#5555FF'>=</font> src_grad;
        gamma_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>gamma<font face='Lucida Console'>)</font>; gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>; gamma_grad2 <font color='#5555FF'>=</font> gamma_grad;
        beta_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>beta<font face='Lucida Console'>)</font>; beta_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>; beta_grad2 <font color='#5555FF'>=</font> beta_grad;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>gradient_input<font face='Lucida Console'>)</font>;


        cpu::<font color='#BB00BB'>batch_normalize_gradient</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>batch_normalize_gradient</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,gradient_input, means, invstds, src, gamma, src_grad2, gamma_grad2, beta_grad2<font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src_grad error:   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>gamma_grad error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>beta_grad error:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='compare_bn_conv_gpu_and_cpu'></a>compare_bn_conv_gpu_and_cpu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor dest, dest2;
        resizable_tensor means, means2;
        resizable_tensor invstds, invstds2;
        resizable_tensor running_means, running_means2;
        resizable_tensor running_variances, running_variances2;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>8</font>,<font color='#979000'>10</font>,<font color='#979000'>9</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        gamma <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        beta <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;

        cpu::<font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest,means,invstds,<font color='#979000'>1</font>,running_means,running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest2,means2,invstds2,<font color='#979000'>1</font>,running_means2,running_variances2, src, gamma, beta<font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>dest error:    </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>means error:   </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>invstds error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>running_means error:   </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>running_variances error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;

        resizable_tensor gradient_input;
        resizable_tensor src_grad, gamma_grad, beta_grad;
        resizable_tensor src_grad2, gamma_grad2, beta_grad2;
        gradient_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>; src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>; src_grad2 <font color='#5555FF'>=</font> src_grad;
        gamma_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>gamma<font face='Lucida Console'>)</font>; gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>; gamma_grad2 <font color='#5555FF'>=</font> gamma_grad;
        beta_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>beta<font face='Lucida Console'>)</font>; beta_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>; beta_grad2 <font color='#5555FF'>=</font> beta_grad;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>gradient_input<font face='Lucida Console'>)</font>;


        cpu::<font color='#BB00BB'>batch_normalize_conv_gradient</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>batch_normalize_conv_gradient</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,gradient_input, means, invstds, src, gamma, src_grad2, gamma_grad2, beta_grad2<font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src_grad error:   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>gamma_grad error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>beta_grad error:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
    <b>}</b>


    <font color='#0000FF'><u>void</u></font> <b><a name='test_more_ops2'></a>test_more_ops2</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        dlib::rand rnd;
        tt::tensor_rand trand;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>100</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            resizable_tensor dest1, dest2, src1, src2;
            src1.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            dest1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            src2.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;

            cpu::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest1, src1, src2<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest2, src1, src2<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest1, src1, src2<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest2, src1, src2<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;


            <font color='#009900'>// now try it using the other mode of multiply_conv
</font>            src2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            dest1.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            dest2.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest1, src1, src2<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest2, src1, src2<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font> scale <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font> scalem <font color='#5555FF'>=</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font> , <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scalem <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font> , <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scalem<font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> prevd2 <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest1, src1, src2<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest2, src1, src2<font face='Lucida Console'>)</font>;
            scale <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            scalem <font color='#5555FF'>=</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>prevd2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font> , <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>prevd2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>prevd2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scalem <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font> , <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>prevd2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scalem<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>100</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            resizable_tensor dest1, dest2, src, A, B;
            src.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            dest1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
            dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
            A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;

            cpu::<font color='#BB00BB'>affine_transform_conv</font><font face='Lucida Console'>(</font>dest1, src, A, B<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>affine_transform_conv</font><font face='Lucida Console'>(</font>dest2, src, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>100</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            resizable_tensor dest1, dest2, g;
            g.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            dest1.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,g.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            dest2.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,g.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>g<font face='Lucida Console'>)</font>;

            cpu::<font color='#BB00BB'>assign_conv_bias_gradient</font><font face='Lucida Console'>(</font>dest1, g<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>assign_conv_bias_gradient</font><font face='Lucida Console'>(</font>dest2, g<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> scale <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> scalem <font color='#5555FF'>=</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font> , <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scalem <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font> , <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scalem<font face='Lucida Console'>)</font>;
        <b>}</b>

    <b>}</b>

<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_USE_CUDA
</font>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_max_pool'></a>test_max_pool</b><font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> window_height,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> window_width,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> stride_y,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> stride_x,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> padding_y,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> padding_x
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor A, B, gradient_input;
        A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>16</font>,<font color='#979000'>7</font><font face='Lucida Console'>)</font>;
        B.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
        gradient_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;

        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>A,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>B,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>gradient_input,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;


        tt::pooling mp;

        mp.<font color='#BB00BB'>setup_max_pooling</font><font face='Lucida Console'>(</font>window_height,window_width,stride_y,stride_x,padding_y,padding_x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>mp</font><font face='Lucida Console'>(</font>A, B<font face='Lucida Console'>)</font>;

        <font color='#009900'>// make sure max pooling does what it's spec says it should.
</font>        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> B.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> B.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>B.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y<font color='#5555FF'>-</font>window_height<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_y<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>B.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x<font color='#5555FF'>-</font>window_width<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_x<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> x_offset <font color='#5555FF'>=</font> window_width<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_x;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> y_offset <font color='#5555FF'>=</font> window_height<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_y;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> s <font color='#5555FF'>=</font> <font color='#979000'>0</font>; s <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>s<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>A,s,k<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>r,c<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>subm_clipped</font><font face='Lucida Console'>(</font><font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>B,s,k<font face='Lucida Console'>)</font>,
                                    <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>c<font color='#5555FF'>*</font>stride_x<font color='#5555FF'>+</font>x_offset,
                                                  r<font color='#5555FF'>*</font>stride_y<font color='#5555FF'>+</font>y_offset,
                                                  window_width,
                                                  window_height<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
                                                  "<font color='#CC0000'>padding: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> padding_x <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> padding_y 
                                                  <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> window size: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> window_width <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> window_height 
                                                  <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> stride_x <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> stride_y
                                                  <font face='Lucida Console'>)</font>;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_avg_pool'></a>test_avg_pool</b><font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> window_height,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> window_width,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> stride_y,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> stride_x,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> padding_y,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> padding_x
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor A, B, gradient_input;
        A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>16</font>,<font color='#979000'>7</font><font face='Lucida Console'>)</font>;
        B.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
        gradient_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;

        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>A,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>B,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>gradient_input,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;


        tt::pooling mp;

        mp.<font color='#BB00BB'>setup_avg_pooling</font><font face='Lucida Console'>(</font>window_height,window_width,stride_y,stride_x,padding_y,padding_x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>mp</font><font face='Lucida Console'>(</font>A, B<font face='Lucida Console'>)</font>;

        <font color='#009900'>// make sure avg pooling does what it's spec says it should.
</font>        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> B.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> B.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>B.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y<font color='#5555FF'>-</font>window_height<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_y<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>B.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x<font color='#5555FF'>-</font>window_width<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_x<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> x_offset <font color='#5555FF'>=</font> window_width<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_x;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> y_offset <font color='#5555FF'>=</font> window_height<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_y;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> s <font color='#5555FF'>=</font> <font color='#979000'>0</font>; s <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>s<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'><u>float</u></font> expected <font color='#5555FF'>=</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>subm_clipped</font><font face='Lucida Console'>(</font><font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>B,s,k<font face='Lucida Console'>)</font>,
                                            <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>c<font color='#5555FF'>*</font>stride_x<font color='#5555FF'>+</font>x_offset,
                                                        r<font color='#5555FF'>*</font>stride_y<font color='#5555FF'>+</font>y_offset,
                                                        window_width,
                                                        window_height<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                        <font color='#0000FF'><u>float</u></font> err <font color='#5555FF'>=</font> <font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>A,s,k<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>r,c<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> expected<font face='Lucida Console'>)</font>;
                        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>err <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>, err <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> expected <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>A,s,k<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>r,c<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_layers'></a>test_layers</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            multiply_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            max_pool_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            avg_pool_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            affine_ <font color='#BB00BB'>l</font><font face='Lucida Console'>(</font>CONV_MODE<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            affine_ <font color='#BB00BB'>l</font><font face='Lucida Console'>(</font>FC_MODE<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            bn_<font color='#5555FF'>&lt;</font>CONV_MODE<font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            bn_<font color='#5555FF'>&lt;</font>FC_MODE<font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            con_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            con_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font>l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            con_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            con_<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            fc_<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>,FC_HAS_BIAS<font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            fc_<font color='#5555FF'>&lt;</font><font color='#979000'>5</font>,FC_HAS_BIAS<font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            fc_<font color='#5555FF'>&lt;</font><font color='#979000'>4</font>,FC_NO_BIAS<font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            relu_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            prelu_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            sig_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            htan_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            softmax_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> n, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> rcon <font color='#5555FF'>=</font> max_pool<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,relu<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font>n,<font color='#979000'>5</font>,<font color='#979000'>5</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> n, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> rfc <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>bn_fc<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font>n,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'><u>void</u></font> <b><a name='test_tagging'></a>test_tagging</b><font face='Lucida Console'>(</font>
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>typedef</font> loss_multiclass_log<font color='#5555FF'>&lt;</font>rfc<font color='#5555FF'>&lt;</font><font color='#979000'>10</font>,skip1<font color='#5555FF'>&lt;</font>rfc<font color='#5555FF'>&lt;</font><font color='#979000'>84</font>,rfc<font color='#5555FF'>&lt;</font><font color='#979000'>120</font>,tag1<font color='#5555FF'>&lt;</font>rcon<font color='#5555FF'>&lt;</font><font color='#979000'>16</font>,rcon<font color='#5555FF'>&lt;</font><font color='#979000'>6</font>,input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>char</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> net_type;

        net_type net;
        net_type <font color='#BB00BB'>net2</font><font face='Lucida Console'>(</font><font color='#BB00BB'>num_fc_outputs</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.num_computational_layers <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>skip1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.num_computational_layers <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>8</font><font color='#5555FF'>+</font><font color='#979000'>3</font><font color='#5555FF'>+</font><font color='#979000'>3</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.num_layers <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>skip1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.num_layers <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>10</font><font color='#5555FF'>+</font><font color='#979000'>3</font><font color='#5555FF'>+</font><font color='#979000'>3</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>layer<font color='#5555FF'>&lt;</font>skip1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>layer<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>layer<font color='#5555FF'>&lt;</font>skip1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>layer<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>net.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_num_outputs</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>net2.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_num_outputs</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>int</u></font> N, 
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='BN'></a>BN</b>, 
        <font color='#0000FF'><u>int</u></font> stride, 
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font> 
    <font color='#0000FF'>using</font> block  <font color='#5555FF'>=</font> BN<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font>N,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,relu<font color='#5555FF'>&lt;</font>BN<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font>N,<font color='#979000'>3</font>,<font color='#979000'>3</font>,stride,stride,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font>,<font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font>,<font color='#0000FF'><u>int</u></font>,<font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='block'></a>block</b>, 
        <font color='#0000FF'><u>int</u></font> N, 
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='BN'></a>BN</b>, 
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> residual <font color='#5555FF'>=</font> add_prev1<font color='#5555FF'>&lt;</font>block<font color='#5555FF'>&lt;</font>N,BN,<font color='#979000'>1</font>,tag1<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font>,<font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font>,<font color='#0000FF'><u>int</u></font>,<font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='block'></a>block</b>, 
        <font color='#0000FF'><u>int</u></font> N, 
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='BN'></a>BN</b>, 
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> residual_down <font color='#5555FF'>=</font> add_prev2<font color='#5555FF'>&lt;</font>avg_pool<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,skip1<font color='#5555FF'>&lt;</font>tag2<font color='#5555FF'>&lt;</font>block<font color='#5555FF'>&lt;</font>N,BN,<font color='#979000'>2</font>,tag1<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> res       <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>residual<font color='#5555FF'>&lt;</font>block,<font color='#979000'>8</font>,bn_con,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> ares      <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>residual<font color='#5555FF'>&lt;</font>block,<font color='#979000'>8</font>,affine,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> res_down  <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>residual_down<font color='#5555FF'>&lt;</font>block,<font color='#979000'>8</font>,bn_con,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> ares_down <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>residual_down<font color='#5555FF'>&lt;</font>block,<font color='#979000'>8</font>,affine,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> 
    <font color='#0000FF'>using</font> pres  <font color='#5555FF'>=</font> prelu<font color='#5555FF'>&lt;</font>add_prev1<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>8</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,prelu<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>8</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,tag1<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'><u>void</u></font> <b><a name='test_visit_funcions'></a>test_visit_funcions</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> net_type2 <font color='#5555FF'>=</font> loss_multiclass_log<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>10</font>,
            avg_pool_everything<font color='#5555FF'>&lt;</font>
            pres<font color='#5555FF'>&lt;</font>res<font color='#5555FF'>&lt;</font>res<font color='#5555FF'>&lt;</font>res_down<font color='#5555FF'>&lt;</font> <font color='#009900'>// 2 prelu layers here
</font>            tag4<font color='#5555FF'>&lt;</font>repeat<font color='#5555FF'>&lt;</font><font color='#979000'>9</font>,pres,    <font color='#009900'>// 9 groups, each containing 2 prelu layers  
</font>            res_down<font color='#5555FF'>&lt;</font>
            res<font color='#5555FF'>&lt;</font>
            input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>char</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>
            <font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        net_type2 pnet;

        <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>pnet.num_layers <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>131</font>, pnet.num_layers<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>pnet.num_computational_layers <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>109</font>, pnet.num_computational_layers<font face='Lucida Console'>)</font>;

        std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>bool</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>hit</font><font face='Lucida Console'>(</font>pnet.num_computational_layers, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'><u>size_t</u></font> count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>visit_layer_parameter_gradients</font><font face='Lucida Console'>(</font>pnet, [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i, tensor<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font><b>{</b>hit[i] <font color='#5555FF'>=</font> <font color='#979000'>true</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>count; <b>}</b><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font> x : hit<font face='Lucida Console'>)</font>
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>count <font color='#5555FF'>=</font><font color='#5555FF'>=</font> pnet.num_computational_layers<font face='Lucida Console'>)</font>;

        count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>bool</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>hit2</font><font face='Lucida Console'>(</font>pnet.num_computational_layers, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>visit_layer_parameters</font><font face='Lucida Console'>(</font>pnet, [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i, tensor<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font><b>{</b>hit2[i] <font color='#5555FF'>=</font> <font color='#979000'>true</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>count; <b>}</b><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font> x : hit2<font face='Lucida Console'>)</font>
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>count <font color='#5555FF'>=</font><font color='#5555FF'>=</font> pnet.num_computational_layers<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>float</u></font> <b><a name='tensor_read_cpu'></a>tensor_read_cpu</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> t, <font color='#0000FF'><u>long</u></font> i, <font color='#0000FF'><u>long</u></font> k, <font color='#0000FF'><u>long</u></font> r, <font color='#0000FF'><u>long</u></font> c<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> p <font color='#5555FF'>=</font> t.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> t.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> t.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> i <font color='#5555FF'>+</font>
                        t.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> t.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> k <font color='#5555FF'>+</font> t.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> r <font color='#5555FF'>+</font> c;
        <font color='#0000FF'>return</font> <font color='#5555FF'>*</font>p;
    <b>}</b>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_copy_tensor_cpu'></a>test_copy_tensor_cpu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>9</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src1</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>3</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src2</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>3</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src3</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>9</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src3<font face='Lucida Console'>)</font>;

        cpu::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>dest, <font color='#979000'>0</font>, src1, <font color='#979000'>0</font>,  src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <font color='#009900'>//full copy src1-&gt;dest
</font>        cpu::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>dest, src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src2, <font color='#979000'>0</font>,  src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <font color='#009900'>//full copy src2-&gt;dest with offset of src1
</font>        cpu::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>dest, src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src3, <font color='#979000'>3</font>,  <font color='#979000'>3</font><font face='Lucida Console'>)</font>; <font color='#009900'>//partial copy src3 into the rest place of dest
</font>

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'><u>float</u></font> dest_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>dest, i, k, r, c<font face='Lucida Console'>)</font>;
                        <font color='#009900'>// first part is from src1
</font>                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src1, i, k, r, c<font face='Lucida Console'>)</font>;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>=</font><font color='#5555FF'>=</font> dest_value<font face='Lucida Console'>)</font>;
                        <b>}</b>
                        <font color='#009900'>// second part is from src2
</font>                        <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src2, i, k <font color='#5555FF'>-</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, r, c<font face='Lucida Console'>)</font>;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>=</font><font color='#5555FF'>=</font> dest_value<font face='Lucida Console'>)</font>;
                        <b>}</b>
                        <font color='#009900'>// third part is from src3
</font>                        <font color='#0000FF'>else</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src3, i, k <font color='#5555FF'>-</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font>, r, c<font face='Lucida Console'>)</font>;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>=</font><font color='#5555FF'>=</font> dest_value<font face='Lucida Console'>)</font>;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>
    <b>}</b>
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
    <font color='#0000FF'><u>void</u></font> <b><a name='test_copy_tensor_gpu'></a>test_copy_tensor_gpu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>9</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src1</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>3</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src2</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>3</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src3</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>9</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src3<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>dest, <font color='#979000'>0</font>, src1, <font color='#979000'>0</font>,  src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <font color='#009900'>//full copy src1-&gt;dest
</font>        cuda::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>dest, src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src2, <font color='#979000'>0</font>,  src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <font color='#009900'>//full copy src2-&gt;dest with offset of src1
</font>        cuda::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>dest, src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src3, <font color='#979000'>3</font>,  <font color='#979000'>3</font><font face='Lucida Console'>)</font>; <font color='#009900'>//partial copy src3 into the rest place of dest
</font>

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'><u>float</u></font> dest_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>dest, i, k, r, c<font face='Lucida Console'>)</font>;
                        <font color='#009900'>// first part is from src1
</font>                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src1, i, k, r, c<font face='Lucida Console'>)</font>;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>=</font><font color='#5555FF'>=</font> dest_value<font face='Lucida Console'>)</font>;
                        <b>}</b>
                            <font color='#009900'>// second part is from src2
</font>                        <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src2, i, k <font color='#5555FF'>-</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, r, c<font face='Lucida Console'>)</font>;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>=</font><font color='#5555FF'>=</font> dest_value<font face='Lucida Console'>)</font>;
                        <b>}</b>
                            <font color='#009900'>// third part is from src3
</font>                        <font color='#0000FF'>else</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src3, i, k <font color='#5555FF'>-</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font>, r, c<font face='Lucida Console'>)</font>;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>=</font><font color='#5555FF'>=</font> dest_value<font face='Lucida Console'>)</font>;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>
    <b>}</b>
<font color='#0000FF'>#endif</font><font color='#009900'>//DLIB_USE_CUDA
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> concat_block1 <font color='#5555FF'>=</font> con<font color='#5555FF'>&lt;</font><font color='#979000'>5</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> concat_block2 <font color='#5555FF'>=</font> con<font color='#5555FF'>&lt;</font><font color='#979000'>8</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> concat_block3 <font color='#5555FF'>=</font> max_pool<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> concat_incept <font color='#5555FF'>=</font> inception3<font color='#5555FF'>&lt;</font>concat_block1,concat_block2,concat_block3,SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'><u>void</u></font> <b><a name='test_concat'></a>test_concat</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> concat_incept<font color='#5555FF'>&lt;</font>input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        resizable_tensor <font color='#BB00BB'>data</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>1</font>, <font color='#979000'>111</font>, <font color='#979000'>222</font><font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;

        net_type net;


        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> out <font color='#5555FF'>=</font> net.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b1o <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>itag1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b2o <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>itag2<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b3o <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>itag3<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>14</font>, <font color='#979000'>111</font>, <font color='#979000'>222</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>dest, <font color='#979000'>0</font>, b1o, <font color='#979000'>0</font>,  b1o.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>dest, b1o.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b2o, <font color='#979000'>0</font>,  b2o.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>dest, b1o.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> b2o.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b3o, <font color='#979000'>0</font>,  b3o.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> out.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'><u>int</u></font> error <font color='#5555FF'>=</font> <font color='#BB00BB'>memcmp</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, out.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>error <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;

        resizable_tensor <font color='#BB00BB'>gr</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>14</font>, <font color='#979000'>111</font>, <font color='#979000'>222</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>gr<font face='Lucida Console'>)</font>;

        resizable_tensor params;
        net.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>backward</font><font face='Lucida Console'>(</font>gr, net, params<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b1g <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>itag1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b2g <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>itag2<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b3g <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>itag3<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        resizable_tensor <font color='#BB00BB'>g1</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>5</font>, <font color='#979000'>111</font>, <font color='#979000'>222</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>g2</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>8</font>, <font color='#979000'>111</font>, <font color='#979000'>222</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>g3</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>1</font>, <font color='#979000'>111</font>, <font color='#979000'>222</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>g1, <font color='#979000'>0</font>, gr, <font color='#979000'>0</font>,  g1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>g2, <font color='#979000'>0</font>, gr, g1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>g3, <font color='#979000'>0</font>, gr, g1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> g2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g3.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>g1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> b1g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        error <font color='#5555FF'>=</font> <font color='#BB00BB'>memcmp</font><font face='Lucida Console'>(</font>g1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b1g.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b1g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>error <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>g2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> b2g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        error <font color='#5555FF'>=</font> <font color='#BB00BB'>memcmp</font><font face='Lucida Console'>(</font>g2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b2g.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b2g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>error <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>g3.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> b3g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        error <font color='#5555FF'>=</font> <font color='#BB00BB'>memcmp</font><font face='Lucida Console'>(</font>g3.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b3g.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b3g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>error <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
    <b>}</b>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='dnn_tester'></a>dnn_tester</b> : <font color='#0000FF'>public</font> tester
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='dnn_tester'></a>dnn_tester</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> :
            tester <font face='Lucida Console'>(</font>"<font color='#CC0000'>test_dnn</font>",
                "<font color='#CC0000'>Runs tests on the deep neural network tools.</font>"<font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='run_tests'></a>run_tests</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// make the tests repeatable
</font>            <font color='#BB00BB'>srand</font><font face='Lucida Console'>(</font><font color='#979000'>1234</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>test_tagging</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            <font color='#BB00BB'>test_conv</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops2</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>3</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops</font><font face='Lucida Console'>(</font><font color='#979000'>10000</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>compare_bn_gpu_and_cpu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>compare_bn_conv_gpu_and_cpu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_add</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>compare_adam</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_copy_tensor_gpu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>40</font>,<font color='#979000'>50</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font>,<font color='#979000'>3</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font>,<font color='#979000'>3</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>40</font>,<font color='#979000'>50</font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_tanh</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_softmax</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_sigmoid</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_batch_normalize</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_batch_normalize_conv</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_basic_tensor_ops</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_layers</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_visit_funcions</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_copy_tensor_cpu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_concat</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='perform_test'></a>perform_test</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>NOW RUNNING TESTS WITH set_dnn_prefer_fastest_algorithms()</font>";
            <font color='#BB00BB'>set_dnn_prefer_fastest_algorithms</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>run_tests</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>NOW RUNNING TESTS WITH set_dnn_prefer_smallest_algorithms()</font>";
            <font color='#BB00BB'>set_dnn_prefer_smallest_algorithms</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>run_tests</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b> a;
<b>}</b>



</pre></body></html>